#/bin/bash
Cur_day=`date -d '6 day ago' +%Y%m%d`
#运行时传城市参数进来 $1
city_today=$1"_$Cur_day"
PATH=`pwd`
Local_inputdata_path=$PATH/$1
Local_code_path=$PATH
Local_hadoop_jar=/usr/local/apps/hadoop-2.6.4/share/hadoop/tools/lib/hadoop-streaming-2.6.4.jar
start_time=`date +s`;
python dumpdata.py $2 $1 
python dumpborough.py $1 $2
python dump_pricelist.py $1 $3
hadoop fs -rmr /ETL/sell/$city_today/*
hadoop fs -mkdir /ETL/sell/$city_today
hadoop fs -copyFromLocal $Local_inputdata_path/house_sell_gov /ETL/sell/$city_today
hadoop fs -copyFromLocal $Local_inputdata_path/heb_jiaoyang /ETL/sell/$city_today
#hadoop fs -copyFromLocal $Local_inputdata_path/house_sell_gov_test1 /ETL/sell/$city_today
mkdir $Local_inputdata_path/$city_today
mkdir $Local_inputdata_path/$city_today/gov_json
mkdir $Local_inputdata_path/$city_today/house_sell_gov
mkdir $Local_inputdata_path/$city_today/house_sell_bad
mkdir $Local_inputdata_path/$city_today/house_sell_del
mkdir $Local_inputdata_path/$city_today/broker_check
hadoop fs -rmr /ETL/sell/$city_today/mapred1
hadoop jar $Local_hadoop_jar \
-libjars "./TestMultipleOutputFormat.jar" \
-input /ETL/sell/$city_today/heb_jiaoyang \
-output /ETL/sell/$city_today/mapred1 \
-outputformat com.zhugefang.hadoop.TestMultipleOutputFormat \
-jobconf suffix.multiple.outputformat.separator="#" \
-mapper '/usr/bin/php mapper1.php' \
-reducer '/usr/bin/php reducer1.php' \
-file "$Local_code_path/mapper1.php" \
-file "$Local_code_path/reducer1.php" \
-file "$Local_code_path/sell.php" \
-file "$Local_code_path/rent.php" \
-file "$Local_code_path/function.php" \
-file "$Local_inputdata_path/borough" \
-file "$Local_inputdata_path/borough_price" \
-file "$Local_inputdata_path/broker"

hadoop fs -rmr /ETL/sell/$city_today/mapred2

hadoop jar $Local_hadoop_jar \
-libjars "./TestMultipleOutputFormat.jar" \
-D map.output.key.field.separator=. \
-D num.key.fields.for.partition=1 \
-partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner \
-input /ETL/sell/$city_today/mapred1/part*-N \
-output /ETL/sell/$city_today/mapred2 \
-outputformat com.zhugefang.hadoop.TestMultipleOutputFormat \
-jobconf suffix.multiple.outputformat.separator="#" \
-mapper '/usr/bin/php mapper2.php' \
-reducer '/usr/bin/php reducer2.php' \
-file "$Local_code_path/mapper2.php" \
-file "$Local_code_path/reducer2.php" \
-file "$Local_code_path/sell.php" \
-file "$Local_code_path/rent.php" \
-file "$Local_code_path/function.php" \

hadoop fs -rmr /ETL/sell/$city_today/mapred4
hadoop jar $Local_hadoop_jar \
-libjars "./TestMultipleOutputFormat.jar" \
-input /ETL/sell/$city_today/mapred1/part-*-B \
-input /ETL/sell/$city_today/mapred2/part-*-B \
-output /ETL/sell/$city_today/mapred4 \
-outputformat com.zhugefang.hadoop.TestMultipleOutputFormat \
-jobconf suffix.multiple.outputformat.separator="#" \
-mapper 'cat' \
-reducer '/usr/bin/php reducer4.php' \
-file "$Local_code_path/reducer4.php" \
-file "$Local_code_path/sell.php" \
-file "$Local_code_path/rent.php" \
-file "$Local_code_path/function.php" 
hadoop fs -rmr /ETL/sell/$city_today/mapred5
hadoop jar $Local_hadoop_jar \
-D mapreduce.job.reduces=0 \
-input /ETL/sell/$city_today/mapred2/part-*-E \
-output /ETL/sell/$city_today/mapred5 \
-mapper '/usr/bin/php mapper5.php' \
-file "$Local_code_path/mapper5.php" \
-file "$Local_code_path/function.php"
rm -rf ./$1/$city_today/house_sell_gov/*
hadoop fs -copyToLocal /ETL/sell/$city_today/mapred5/* ./$1/$city_today/house_sell_gov
hadoop fs -getmerge /ETL/sell/$city_today/mapred5/part* ./$1/$city_today/house_sell_gov/house_sell_gov
hadoop fs -rmr /ETL/sell/$city_today/mapred6
hadoop jar $Local_hadoop_jar \
-D mapreduce.job.reduces=0 \
-input /ETL/sell/$city_today/mapred4/part* \
-output /ETL/sell/$city_today/mapred6 \
-mapper '/usr/bin/php mapper6.php' \
-file "$Local_code_path/mapper6.php" \
-file "$Local_code_path/function.php"
rm -rf ./$1/$city_today/house_sell_bad/*
hadoop fs -copyToLocal /ETL/sell/$city_today/mapred6/* ./$1/$city_today/house_sell_bad
hadoop fs -getmerge /ETL/sell/$city_today/mapred6/part* ./$1/$city_today/house_sell_bad/house_sell_bad
hadoop fs -rmr /ETL/sell/$city_today/mapred7
hadoop jar $Local_hadoop_jar \
-D mapreduce.job.reduces=0 \
-input /ETL/sell/$city_today/mapred1/part-*-D \
-output /ETL/sell/$city_today/mapred7 \
-mapper '/usr/bin/php mapper7.php' \
-file "$Local_code_path/mapper7.php" \
-file "$Local_code_path/function.php"
rm -rf ./$1/$city_today/house_sell_del/*
hadoop fs -copyToLocal /ETL/sell/$city_today/mapred7/* ./$1/$city_today/house_sell_del
hadoop fs -getmerge /ETL/sell/$city_today/mapred7/part* ./$1/$city_today/house_sell_del/house_sell_del

hadoop fs -rmr /ETL/sell/$city_today/mapred8
hadoop jar $Local_hadoop_jar \
-D mapreduce.job.reduces=0 \
-input /ETL/sell/$city_today/mapred1/part-*-C \
-output /ETL/sell/$city_today/mapred8 \
-mapper '/usr/bin/php mapper8.php' \
-file "$Local_code_path/mapper8.php" \
-file "$Local_code_path/function.php"
rm -rf ./$1/$city_today/broker_check/*
hadoop fs -copyToLocal /ETL/sell/$city_today/mapred8/* ./$1/$city_today/broker_check
hadoop fs -getmerge /ETL/sell/$city_today/mapred8/part* ./$1/$city_today/broker_check/broker_check
hadoop fs -rmr /ETL/sell/$city_today/mapred9
hadoop jar $Local_hadoop_jar \
-D mapreduce.job.reduces=0 \
-input /ETL/sell/$city_today/mapred2/part-*-E \
-output /ETL/sell/$city_today/mapred9 \
-mapper '/usr/bin/php mapper9.php' \
-file "$Local_code_path/mapper9.php" \
-file "$Local_code_path/function.php"
rm -rf ./$1/$city_today/gov_json/*
hadoop fs -copyToLocal /ETL/sell/$city_today/mapred9/* ./$1/$city_today/gov_json
hadoop fs -getmerge /ETL/sell/$city_today/mapred9/part* ./$1/$city_today/gov_json/gov_json

end_time=`date +s`
echo "time spended\n";
echo $(($end_time-$start_time))
